# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
from bs4 import BeautifulSoup as bs
from dingdian.spiders.cnvd import  Myspider as cnvdSpider
from dingdian.spiders.seebug import  Myspider as seeBugSpider
from dingdian.spiders.cnnvd import Myspider as cnnvdSpider
from dingdian.spiders.exploit import Myspider as exploitSpider
import time
from dingdian.mySelenium import MySelenium

class DingdianSpiderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.


        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.
        # Must return an iterable of Request, dict or Item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.
        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class DingdianDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        if isinstance(spider, seeBugSpider):
            spider.download_delay=6
        if isinstance(spider, exploitSpider):
           spider.download_delay=2
        # Called for each request that goes through the downloader
        # middleware.
        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    def process_response(self, request, response, spider):

        if response.status == 403:
            time.sleep(1)
            print "403错误", response.url
            return request
        if isinstance(spider,cnvdSpider):
            #print request.meta.get('depth')
            if request.meta.get('flag') ==1:
                #print response.url+"**"+request.url
                pageSource = response.text
                try:
                    title = bs(pageSource, "lxml").find("h1").get_text()

                except Exception as e:
                    print "中间件错误"
                    mySelenium = MySelenium()
                    cnvdSpider.cookieDic = mySelenium.getCookieAndSource(response.url, "chrome")[1]
                    request.cookies = cnvdSpider.cookieDic
                    #print cnvdSpider.cookieDic
                    return request
        if isinstance(spider,seeBugSpider):

            #print request.meta.get('depth')
            if request.meta.get('flag') ==1:
                #print response.url+"**"+request.url
                pageSource = response.text
                try:
                    title = bs(pageSource, "lxml").find("h1").get_text()

                except Exception as e:
                    print "中间件错误"
                    print response.url
                    mySelenium=MySelenium()
                    seeBugSpider.cookieDic = mySelenium.getCookieAndSource(response.url,"chrome")[1]
                    request.cookies = seeBugSpider.cookieDic
                    return request
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain

        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


# import random, base64
#
#
# class ProxyMiddleware(object):
#     proxyList = [ '195.53.114.105:8080'   ]
#
#     def process_request(self, request, spider):
#         # Set the location of the proxy
#         pro_adr = random.choice(self.proxyList)
#         print "USE PROXY -> " + pro_adr
#         request.meta['proxy'] = "http://" + pro_adr

